{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 构造特征"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "构造user_pop,item_pop,user_rate,item_rate,lfm_reco"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from lightgbm.sklearn import LGBMClassifier\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dpath = \"./data/\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.构造lfm_reco"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lfm_train(train_data, F, alpha, beta, step):\n",
    "    \"\"\"\n",
    "    train LFM model,get latent factor user_vec and item_vec\n",
    "    Args:\n",
    "        train_data: train_data for lfm\n",
    "        F: user vector len, item vector len\n",
    "        alpha:regularization factor\n",
    "        beta: learning rate\n",
    "        step: iteration number\n",
    "    Return:\n",
    "        dict: key itemid, value:np.ndarray\n",
    "        dict: key userid, value:np.ndarray\n",
    "    \"\"\"\n",
    "    user_vec = {}\n",
    "    item_vec = {}\n",
    "    count = 0\n",
    "    for step in range(step):\n",
    "        fin = open(dpath+train_data,\"r+\")\n",
    "        start = 0\n",
    "        for line in fin:\n",
    "            if start == 0:\n",
    "                start += 1\n",
    "                continue\n",
    "            cols = line.strip().split(\",\")\n",
    "            userid,itemid,target = cols[0],cols[1],cols[-1]\n",
    "            if userid not in user_vec:\n",
    "                user_vec[userid] = np.random.randn(F)\n",
    "            if itemid not in item_vec:\n",
    "                item_vec[itemid] = np.random.randn(F)\n",
    "            #target是str，需转换为int\n",
    "            delta = int(target)-lfm_score(user_vec[userid],item_vec[itemid])\n",
    "            for i in range(F):\n",
    "                user_vec[userid][i] += beta*(delta*item_vec[itemid][i]\\\n",
    "                                            -alpha*user_vec[userid][i])\n",
    "                item_vec[itemid][i] += beta*(delta*user_vec[userid][i]\\\n",
    "                                            -alpha*item_vec[itemid][i])\n",
    "            count += 1\n",
    "            #每100000个样本更新一次学习率\n",
    "            if count%100000==0:\n",
    "                beta *= 0.95\n",
    "    pickle.dump(user_vec,open(dpath+\"user_vec.pkl\",\"wb\"))\n",
    "    pickle.dump(item_vec,open(dpath+\"item_vec.pkl\",\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def lfm_score(user_vector,item_vector):\n",
    "    \"\"\"\n",
    "    user_vector and item_vector distance\n",
    "    Args:\n",
    "        user_vector: lfm model produce user vector\n",
    "        item_vector: lfm model produce item vector\n",
    "    Return:\n",
    "         lfm recommend score\n",
    "    \"\"\"\n",
    "    res = np.dot(user_vector, item_vector)/\\\n",
    "                (np.linalg.norm(user_vector)*np.linalg.norm(item_vector))\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "lfm_train(\"train_merge.csv\", 30, 0.01, 0.1, 10)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.构造user_pop和item_pop，user_rate和item_rate  \n",
    "user_record = {userid: [value1,value2],...}  \n",
    "item_record = {itemid: [value1,value2],...}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_record_file(input_file):\n",
    "    \"\"\"\n",
    "    get user_record and item_record dict\n",
    "    \"\"\"\n",
    "    fin = open(dpath+input_file,\"r+\")\n",
    "    user_record = {}\n",
    "    item_record = {}\n",
    "    start = 0\n",
    "    for line in fin:\n",
    "        if start == 0:\n",
    "            start += 1\n",
    "            continue\n",
    "        cols = line.strip().split(\",\")\n",
    "        userid,itemid,target = cols[0],cols[1],cols[-1]\n",
    "        if userid not in user_record:\n",
    "            user_record[userid] = [0,0]\n",
    "        user_record[userid][0] += 1\n",
    "        #TypeError: unsupported operand type(s) for +=: 'int' and 'str'  \n",
    "        #int(target)\n",
    "        user_record[userid][1] += int(target)\n",
    "        if itemid not in item_record:\n",
    "            item_record[itemid] = [0,0]\n",
    "        item_record[itemid][0] += 1\n",
    "        item_record[itemid][1] += int(target)\n",
    "    pickle.dump(user_record,open(dpath+\"user_record.pkl\",\"wb\"))\n",
    "    pickle.dump(item_record,open(dpath+\"item_record.pkl\",\"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_record_file(\"train_merge.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_record = pickle.load(open(dpath+\"user_record.pkl\",\"rb\"))\n",
    "item_record = pickle.load(open(dpath+\"item_record.pkl\",\"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(user_record),len(item_record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mean_std(user_record,item_record):\n",
    "    \"\"\"\n",
    "    get user_pop mean and std, get item_pop mean and std\n",
    "    \"\"\"\n",
    "    user_pop_mean = np.mean(list(map(lambda x:x[1],user_record.values())))\n",
    "    user_pop_std = np.std(list(map(lambda x:x[1],user_record.values())))\n",
    "    \n",
    "    item_pop_mean = np.mean(list(map(lambda x:x[1],item_record.values())))\n",
    "    item_pop_std = np.std(list(map(lambda x:x[1],item_record.values())))\n",
    "    \n",
    "    return user_pop_mean,user_pop_std,item_pop_mean,item_pop_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_pop_mean,user_pop_std,item_pop_mean,item_pop_std = get_mean_std(user_record,item_record)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "user_pop_mean,user_pop_std,item_pop_mean,item_pop_std"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.把user_pop,item_pop,user_rate,item_rate,lfm_reco写入文件保存，生成最终训练文件"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "user_rate和item_rate取值为何一样  \n",
    "outcols item_rate写成user_rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_train_final(input_file,output_file):\n",
    "    \"\"\"\n",
    "    generate final train file\n",
    "    \"\"\"\n",
    "    fin = open(dpath+input_file,\"r+\")\n",
    "    fout = open(dpath+output_file,\"w+\")\n",
    "    user_vec = pickle.load(open(dpath+\"user_vec.pkl\",\"rb\"))\n",
    "    item_vec = pickle.load(open(dpath+\"item_vec.pkl\",\"rb\"))\n",
    "    user_record = pickle.load(open(dpath+\"user_record.pkl\",\"rb\"))\n",
    "    item_record = pickle.load(open(dpath+\"item_record.pkl\",\"rb\"))\n",
    "    user_pop_mean,user_pop_std,item_pop_mean,item_pop_std\\\n",
    "                    = get_mean_std(user_record,item_record)\n",
    "    start = 0\n",
    "    lfm_reco = 0\n",
    "    outcols = []\n",
    "    for line in fin:\n",
    "        cols = line.strip().split(\",\")\n",
    "        #写入column name\n",
    "        if start == 0:\n",
    "            outcols = cols[:-1]+[\"user_pop\",\"item_pop\",\"user_rate\",\"item_rate\",\"lfm_reco\"]+[cols[-1]]\n",
    "            fout.write(\",\".join(outcols)+\"\\n\")\n",
    "            start += 1\n",
    "            continue\n",
    "        userid,itemid = cols[0],cols[1]\n",
    "        #计算user_pop，item_pop\n",
    "        user_pop = round((user_record[userid][1]-user_pop_mean)/user_pop_std,3)\n",
    "        item_pop = round((item_record[itemid][1]-item_pop_mean)/item_pop_std,3)\n",
    "        #计算user_rate，item_rate\n",
    "        user_rate = round(user_record[userid][1]/user_record[userid][0],3)\n",
    "        item_rate = round(item_record[itemid][1]/item_record[itemid][0],3)\n",
    "        #计算lfm_reco\n",
    "        if cols[0] in user_vec and cols[1] in item_vec:\n",
    "            lfm_reco = lfm_score(user_vec[cols[0]],item_vec[cols[1]])\n",
    "            lfm_reco = np.around(lfm_reco,decimals=5)\n",
    "            outcols = cols[:-1]+[str(user_pop)]+[str(item_pop)]+\\\n",
    "                      [str(user_rate)]+[str(item_rate)]+[str(lfm_reco)]+[cols[-1]]\n",
    "        else:\n",
    "            continue\n",
    "        #写入文件\n",
    "        fout.write(\",\".join(outcols)+\"\\n\")\n",
    "    fin.close()\n",
    "    fout.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "generate_train_final(\"train_merge.csv\",\"train_final.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>msno</th>\n",
       "      <th>song_id</th>\n",
       "      <th>source_system_tab</th>\n",
       "      <th>source_screen_name</th>\n",
       "      <th>source_type</th>\n",
       "      <th>song_length</th>\n",
       "      <th>genre_ids</th>\n",
       "      <th>language</th>\n",
       "      <th>mult_genre</th>\n",
       "      <th>city</th>\n",
       "      <th>...</th>\n",
       "      <th>gender</th>\n",
       "      <th>registered_via</th>\n",
       "      <th>registration_init_time</th>\n",
       "      <th>expiration_date</th>\n",
       "      <th>user_pop</th>\n",
       "      <th>item_pop</th>\n",
       "      <th>user_rate</th>\n",
       "      <th>item_rate</th>\n",
       "      <th>lfm_reco</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=</td>\n",
       "      <td>BBzumQNXUHKdEBOB7mAJuzok+IJA1c2Ryg/yzTF6tik=</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>-0.25183</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>15.581</td>\n",
       "      <td>0.795</td>\n",
       "      <td>0.506</td>\n",
       "      <td>0.474</td>\n",
       "      <td>0.59038</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>bhp/MpSNoqoxOIB+/l8WPqu6jldth4DIpCm3ayXnJqM=</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>0.23361</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>1.991</td>\n",
       "      <td>-0.081</td>\n",
       "      <td>0.743</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.21555</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>JNWfrrC7zNN7BdMpsISKa4Mw+xVJYNnxXh3/Epw7QgY=</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>-0.13422</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>1.991</td>\n",
       "      <td>-0.072</td>\n",
       "      <td>0.743</td>\n",
       "      <td>0.500</td>\n",
       "      <td>-0.08262</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=</td>\n",
       "      <td>2A87tzfnJTSWqD7gIZHisolhe4DMdzkbd6LzO1KHjNs=</td>\n",
       "      <td>3</td>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>0.05294</td>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>13</td>\n",
       "      <td>1.991</td>\n",
       "      <td>-0.081</td>\n",
       "      <td>0.743</td>\n",
       "      <td>1.000</td>\n",
       "      <td>0.12441</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=</td>\n",
       "      <td>3qm6XTZ6MOCU11x8FIVbAGH5l5uMkT3/ZalWG1oo2Gc=</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>-0.36784</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "      <td>15.581</td>\n",
       "      <td>1.211</td>\n",
       "      <td>0.506</td>\n",
       "      <td>0.364</td>\n",
       "      <td>0.55454</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           msno  \\\n",
       "0  FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=   \n",
       "1  Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "2  Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "3  Xumu+NIjS6QYVxDS4/t3SawvJ7viT9hPKXmf0RtLNx8=   \n",
       "4  FGtllVqz18RPiwJj/edr2gV78zirAiY/9SmYvia+kCg=   \n",
       "\n",
       "                                        song_id  source_system_tab  \\\n",
       "0  BBzumQNXUHKdEBOB7mAJuzok+IJA1c2Ryg/yzTF6tik=                  1   \n",
       "1  bhp/MpSNoqoxOIB+/l8WPqu6jldth4DIpCm3ayXnJqM=                  3   \n",
       "2  JNWfrrC7zNN7BdMpsISKa4Mw+xVJYNnxXh3/Epw7QgY=                  3   \n",
       "3  2A87tzfnJTSWqD7gIZHisolhe4DMdzkbd6LzO1KHjNs=                  3   \n",
       "4  3qm6XTZ6MOCU11x8FIVbAGH5l5uMkT3/ZalWG1oo2Gc=                  1   \n",
       "\n",
       "   source_screen_name  source_type  song_length  genre_ids  language  \\\n",
       "0                   5            4     -0.25183         10         8   \n",
       "1                   6            3      0.23361          4         8   \n",
       "2                   6            3     -0.13422          4         8   \n",
       "3                   6            3      0.05294         17         0   \n",
       "4                   5            4     -0.36784          1         8   \n",
       "\n",
       "   mult_genre  city   ...    gender  registered_via  registration_init_time  \\\n",
       "0           0     1   ...         2               7                       8   \n",
       "1           0    13   ...         0               9                       7   \n",
       "2           0    13   ...         0               9                       7   \n",
       "3           0    13   ...         0               9                       7   \n",
       "4           0     1   ...         2               7                       8   \n",
       "\n",
       "   expiration_date  user_pop  item_pop  user_rate  item_rate  lfm_reco  target  \n",
       "0               13    15.581     0.795      0.506      0.474   0.59038       1  \n",
       "1               13     1.991    -0.081      0.743      1.000   0.21555       1  \n",
       "2               13     1.991    -0.072      0.743      0.500  -0.08262       1  \n",
       "3               13     1.991    -0.081      0.743      1.000   0.12441       1  \n",
       "4               13    15.581     1.211      0.506      0.364   0.55454       1  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_final = pd.read_csv(dpath+\"train_final.csv\")\n",
    "train_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7377403, 21)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_final.shape"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
